<?php
/* 
 * @package Analyser
 * @author Yves Peeters
*/


include_once  "classes/Pdo.php";

/**
 * Class to help with read/write spider database
 * @author Yves Peeters
 */
Class SpiderDB extends DB_PDO {

    

    public function __construct() {
        parent::__construct();
        $this->logger = new Logger ("logs/spiderdblog.txt");
    }

    /**
     * empty crawler table
     * {@source}
     */
    public function initSpider() {
        $query=("truncate table crawler");
        $this->exec($query);
        $this->logger->log("init");
    }


    /**
     * adds a list of urls to the database
     * {@source}
     * @param Array $urls
     * @param integer Id of the page where the urls have been found
     */
    function addUrls($urls,$origin){
        $query = "select id from crawler where url = '$origin'";
        $this->logger->log("select id from crawler where url = $origin");
        $stmt = $this->query($query);
        $obj = $stmt->fetch(PDO::FETCH_OBJ);
        $id = $obj->id;
        $this->beginTransaction();
        foreach ($urls as $url) {
            // ignore is faster than checking if each url already exists
            $this->exec("insert ignore into crawler (url,origin) values ('$url','$id')");
        }
        $this->commit();
    }
    /**
     * returns a link that has not been spidered
     * {@source}
     */
    function getUrlToWorkOn() {
        $query = "select url from crawler where parsed = 0 limit 1";
        $stmt = $this->query($query);
        $obj = $stmt->fetch(PDO::FETCH_OBJ);
        $url = $obj->url;
        $this->setUrlAsDone($url);  
        $this->logger->log("return url to work on $url");
        return $url;
    }

    /**
     * sets a url as done by altering the parsed field
     * {@source}
     */
    function setUrlAsDone($url) {
        $query = "update crawler set parsed = 1 where url = '$url'";
        $this->exec($query);
    }
    /**
     * returns a list of urls that need to be spidered
     * {@source}
     */
    function getUrlsToDo() {
        $query = "select url from crawler where parsed = 0";
        $stmt = $this->query($query);
        $return = array();
        while ($obj = $stmt->fetch(PDO::FETCH_OBJ)) {
            array_push($return, $obj->url);
        }
        return $return;
    }

    /**
     * returns a list of urls that already have been spidered
     * {@source}
     */
    function getUrlsDone() {
        $query = "select url from crawler where parsed = 1";
        $stmt = $this->query($query);
        $return = array();
        while($obj = $stmt->fetch(PDO::FETCH_OBJ)) {
            array_push($return, $obj->url);
        }
        return $return;
    }

    /**
     * returns a list of urls that returned a wrong httpheader
     * {@source}
     */
    function getBadUrls() {
        $query = "select url from crawler where found = 0";
        $stmt = $this->query($query);
        $return = array();
        while($obj = $stmt->fetch(PDO::FETCH_OBJ)) {
            array_push($return, $obj->url);
        }
        return $return;
    }

    /**
     *  returns a list of urls with the page where they were found
     * {@source}
     */
    function getBadUrlsByReferer(){
        $query = "SELECT
            Maincrawler.id,
            Maincrawler.url,
            Maincrawler.origin,
            (select Subcrawler.url from
            crawler as Subcrawler
            where Subcrawler.id = Maincrawler.origin
            ) as result FROM crawler as Maincrawler where Maincrawler.found = 0";
        $stmt = $this->query($query);
        $return = array();
        while($obj = $stmt->fetch(PDO::FETCH_OBJ)) {
            array_push($return, array($obj->url,$obj->result));
        }
        return $return;
    }

    /**
     * Return a list with urls that returned httpresponse 200
     * {@source}
     * @return Array
     */

    function getGoodUrls() {
        $query = "select url from crawler where found = 1";
        $stmt = $this->query($query);
        $return = array();
        while($obj = $stmt->fetch(PDO::FETCH_OBJ)) {
            array_push($return, $obj->url);
        }
        return $return;
    }

    /**
     * Mark url as bad
     * sets the found field to 0
     * {@source}
     * @param String $url
     */
    function setUrlAsBad($url) {
        $query = "update crawler set found = 0 where url = '$url'";
        $this->exec($query);
    }

    /**
     * Mark url as good
     * sets the found field to 1
     * {@source}
     * @param String $url
     */
    function setUrlAsGood($url){
        $this->logger->log("url good: $url");
        $query = "update crawler set found = 1 where url = '$url'";
        $this->exec($query);
    }

    /**
     * returns the amount of urls in the crawler
     * {@source}
     * @return Int
     */
    function countUrls(){
        $query='select * from crawler';
        $stmt = $this->query($query);
        return $stmt->rowCount();
    }

    /**
     * returns the amount of bad urls
     * {@source}
     * @return Int
     */
    function countBadUrls(){
        $query="select * from crawler where found='0'";
        $stmt = $this->query($query);
        return $stmt->rowCount();
    }
    
    function setResponsHeader($header, $url){
        $query = "update crawler set header = $header where url = '$url'";
        $this->exec($query);
    }
    
    function getRedirectedUrls(){
        $query = "select url from crawler where header = '301'";
        $stmt = $this->query($query);
        $return = array();
        while($obj = $stmt->fetch(PDO::FETCH_OBJ)) {
            array_push($return, $obj->url);
        }
        return $return;
    }
}
?>
